/**
 * 
 */
package name.yzhu.converter;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.List;
import java.util.Map;

import name.yzhu.common.net.HttpClientRequest;

import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.Header;
import org.cyberneko.html.parsers.DOMParser;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
/**
 * @author yhzhu
 *
 */
public class HtmlCleaner {
	private static final Log LOG = LogFactory.getLog(HtmlCleaner.class);
	private static String[] removedTags = new String[] { "script", "style" };

	public static Document getXmlDoc(InputSource in) throws ConverterException	{
		DOMParser parser = new DOMParser();	   
		try {			
			parser.parse(in);
			return parser.getDocument();
		} catch (SAXException e) {
			LOG.error(e.getMessage(), e);
			throw new ConverterException(e);
		} catch (IOException e) {
			LOG.error(e.getMessage(), e);
			throw new ConverterException(e);
		}		
	}
	
	public static String converter(Reader reader) throws ConverterException	{
		Document doc=getXmlDoc(new InputSource(reader));
		removeTags(doc);
		HtmlNode ivn=rebuildTree(doc);
        ivn.processEvaluate();
        StringBuffer sb=new StringBuffer(2000);
        HtmlNode node= ivn.getHighestNode();
        List<HtmlNode> brothers=node.findBrotherIncludeSelf();
        for(HtmlNode dn:brothers){
           dn.removeGabageTail().getPlainString(sb);
           sb.append("\n\n\n\n\n");
        }
        String result=sb.toString();
        result=refineHeadAndTail(result);
        result=result.replaceAll("\\n\\s*", "\n").replaceAll("\\n\\n\\n+", "\\n\\n\\n");
        if(result.length()<240 && result.matches("(?is).*?copyright.*")){
     	   result="";
        }
        return result.trim();
	}

	public static String converter(String txt) throws ConverterException	{
		StringReader reader = new StringReader(txt);
		try {
		  return converter(reader);
		}
		finally {
		  reader.close();
		}
	}
	
	public static String converterFromWeb(HttpClientRequest client, String url, String defaultCharset) throws ConverterException {
		try{		
			Pair<InputStream, Header[]> web = client.getWebStream(url);
			Map<String, String> headers = HttpClientRequest.headersToMap(web.getRight());
			String charset = headers.get(HttpClientRequest.CONTENT_CHARSET);
			if (null == charset)
				charset = defaultCharset;
			Reader reader = new InputStreamReader(web.getLeft(), charset);
			
			return converter(reader);
		} catch (Exception e) {
			LOG.error(e.getMessage(), e);
			throw new ConverterException(e);
		}	
	}

	public static String converterFromWeb(HttpClientRequest client, String url) throws ConverterException {
		return converterFromWeb(client, url, name.yzhu.common.Constant.UTF_8_STRING);
	}
	
	public static String converterFromWeb(String url) throws ConverterException {
		HttpClientRequest client = new HttpClientRequest();
		try {
		  client.open();
		  return converterFromWeb(client , url, name.yzhu.common.Constant.UTF_8_STRING);
		}
		finally {
			client.close();
		}
	}
	
	private static HtmlNode rebuildTree(Node node) {
		HtmlNode root=new HtmlNode(null,null);
	    if(node.getNodeType()==Node.ELEMENT_NODE)
	    	root.setName(node.getNodeName());
	    else if(node.getNodeType()==Node.DOCUMENT_NODE)
	    	root.setName("root");
	    else if(node.getNodeType()==Node.TEXT_NODE )   {
	    	root.setLeaf(true);
	    	String value=node.getNodeValue();
	    	if(!value.trim().equals(""))	{
	    	   root.setValue(value);
	    	   return root;
	    	}
	    	else
	    		return null;
	    }
	    else
	    	return null;
	    Node child = node.getFirstChild();
	    while (child != null) {
	    	HtmlNode ivn=rebuildTree(child);
	    	if(ivn!=null)
			root.addNode(ivn);
			child=child.getNextSibling();
		}
	    return root;
		
	}

	private static boolean isNeedRemoved(String name) {
		for (String s : removedTags)
			if (name.equalsIgnoreCase(s))
				return true;
		return false;
	}

	private static void removeTags(Node node) {
		Node child = node.getFirstChild();
		while (child != null) {
			Node nextChild = child.getNextSibling();
			if (isNeedRemoved(child.getNodeName()) || child.getNodeType()==Node.ATTRIBUTE_NODE
					|| child.getNodeType()==Node.COMMENT_NODE
					|| child.getNodeType()==Node.CDATA_SECTION_NODE) {
				node.removeChild(child);
				child = nextChild;
				continue;
			} else {
				removeTags(child);
				child = child.getNextSibling();
			}
		}
	}
	
	private static String refineHeadAndTail(String text){
		String multiEmptyLine="\n\n\n\n\n";
		String[] allseg=text.split(multiEmptyLine);
		if(allseg==null || allseg.length==0){
			return "";
		}
		int largest=0;
		int headpoint=0;
		int tailpoint=allseg.length;
		int pos=0;
				
		for(int i=0;i<allseg.length;i++){
			String s=allseg[i];
			if(s.length()>largest){
				largest=s.length();
				pos=i;
			}
		
		}
		
		boolean changed=true;		
		while(changed){
			changed=false;
			if(allseg[headpoint].length()<50 || allseg[headpoint].length()<largest/2 ||
					(int)(allseg[headpoint].length()/allseg[headpoint].split("\n").length)<5 ){
				
				
			   if(headpoint!=pos){
				   headpoint++;
				   changed=true;
			   }
			}
			

				if(allseg[tailpoint-1].length()<50 || allseg[tailpoint-1].length()<largest/2 ||
						(int)(allseg[tailpoint-1].length()/allseg[tailpoint-1].split("\n").length)<5 ){
					
					
				   if(tailpoint-1!=pos){
					   tailpoint--;
					   changed=true;
				   }
				}
		
		}
		
		StringBuilder sb=new StringBuilder();
		for(int i=headpoint;i<tailpoint;i++){
		    sb.append(allseg[i]);
		    sb.append(multiEmptyLine);
		}
		return sb.toString();
		
	}
}
